package com.ajaxw.main;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.RandomAccessFile;
import java.io.StringReader;
import java.io.Writer;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Scanner;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;
import org.jsoup.helper.StringUtil;

import com.ajaxw.analyser.Indexer;
import com.ajaxw.analyser.InvertedLists;
import com.ajaxw.consts.STR;
import com.ajaxw.crawler.Crawler;
import com.ajaxw.crawler.Dispatcher;
import com.ajaxw.crawler.JobInfo;
import com.ajaxw.database.DBAccess;

public class Main {

	/**
	 * @param args
	 * @throws IOException
	 * @throws MalformedURLException
	 */
	private static String data_dir = "E:\\workspace\\data";

	public static void main(String[] args) throws IOException,
			ClassNotFoundException, SQLException {
		// DBAccess db = new DBAccess();
		// db.execute("update webpage set text='你哈1奥123' where lineNo=1");
		// String str = "你好";
		// String strGbk = new String(str.getBytes("utf8"), "gbk");
		// String strUtf8 = new String(str.getBytes(), "utf8");
		// System.out.println(strGbk);
		// System.out.println(strUtf8);
		// System.out.println(Charset.defaultCharset());
		// byte[] bs = str.getBytes("unicode");
		// byte[] bgbk = str.getBytes("gbk");
		// byte[] butf8 = str.getBytes("utf8");
		//
		// for (byte b : bs) {
		// System.out.println(b);
		// }
		//
		// System.out.println("---------------------------");
		//
		// for (byte b : bgbk) {
		// System.out.println(b);
		// }
		//
		// System.out.println("---------------------------");
		//
		// for (byte b : butf8) {
		// System.out.println(b);
		// }
		// String str = new String("你好".getBytes("gbk"), "gbk");
		// System.out.println(str);
		// DBAccess db = new DBAccess();
		// db.execute(String.format("update webpage set text='%s' where lineNo=1",
		// str));
		// System.out.println("...");
		// System.out.println(Charset.defaultCharset());
		//

		Main m = new Main();
		// m.dumpDistribution();

		// m.checkChineseSimple(-1);
		//
		// // // m.insertQueryLog(602999);
		// m.crawler(0);
		// m.update_rcd_of_id(2580);
		// m.multithreadCrawler(12000);

		// m.CreateIndex(-1);
		// m.mergeInvtFiles();
		// m.dumpFile("e:\\tmp\\invt_lst_00000001.dat");
		m.dumpFile("e:\\tmp\\1309029204708.dat");

		// Crawler c = new Crawler();
		// String url =
		// "http://www.efu.com.cn/data/2006/2006-01-13/134496.shtml";
		// String url = "http://www.tudou.com/programs/view/wQWI97Rp6Sg/";
		// String url = "http://www.zigui.org/bbs/read.php?tid=138820";
		// String url =
		// "http://dujijishu.com/redirect.php?fid=8&tid=416&goto=nextnewset.";
		//
		// Map<String, String> rst = c.getPage(url);
		// System.out.println(rst.get("enc"));
		// System.out.println(rst.get("txt"));
		// System.out.println(rst.get("rsp"));

		/* detect Chinese Simplified */
		// System.out.println(Runtime.getRuntime().totalMemory() / 1024 / 1024);
		// DBAccess db = new DBAccess();
		// String str = "芩飪我";
		//
		// ResultSet rs =
		// db.executeQuery("select * from webpage where id=2370");
		// if (rs.next()) {
		// str = rs.getString("text");
		// // str = new String(str.getBytes("utf-8"), "unicode");
		// }
		// String code = StringEscapeUtils.escapeJava(str);
		// System.out.println(str);
		// System.out.println(code);
		// Pattern pattern = Pattern.compile("[^\u4e00-\u9fa5]+");
		// Matcher matcher = null;
		// matcher = pattern.matcher(str);
		// while (matcher.find()) {
		// System.out.println(matcher.group());
		// System.out.println("aaa");
		// }

		// int cnt = 0;
		// for (int i = '\u4e00'; i < '\u9fa5'; i++) {
		// if (i == '啊')
		// System.out.println(String.format("%x", i));
		// // cnt++;
		// // if (cnt % 50 == 0)
		// // System.out.println();
		// // System.out.print((char) i);
		// }
		// System.out.println("total: " + cnt);
		// System.out.println('\u554a');
		// System.out.println('\ub0a1');

	}

	public void dumpDistribution() throws IOException {
		File fin = new File("E:\\tmp\\1309029204708.dat");
		RandomAccessFile raf = new RandomAccessFile(fin, "r");
		File fout = new File("E:\\tmp\\distribution.dat");
		DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(
				new FileOutputStream(fout)));

		int c, l;

		while (true) {
			try {
				c = raf.readInt();
				l = raf.readInt();
			} catch (EOFException eof) {
				break;
			}

			System.out.print((char) c);
			System.out.println(l);

			raf.skipBytes((l + 1) * Integer.SIZE);
		}

		raf.close();
		dos.close();
	}

	public void dumpFile(String fileName) throws IOException {
		DataInputStream dis = new DataInputStream(new BufferedInputStream(
				new FileInputStream(fileName)));

		boolean eof = false;
		int c = -1;
		int l = -1;
		int cnt = 0;
		while (!eof) {
			try {
				c = dis.readInt();
				l = dis.readInt();
			} catch (EOFException e) {
				eof = true;
				break;
			}
			System.out.println((char) c);
			System.out.println("len: " + l);
			for (int j = 0; j < l; j++) {
				if (j != 0 && j % 10 == 0)
					dis.readInt();// System.out.println();
				// System.out.print(String.format("%10d", dis.readInt()));
			}

			System.out.println();
			System.out.println();

			cnt++;
			if (cnt > 50) {
				break;
			}
		}
		dis.close();
	}

	public void mergeInvtFiles() throws IOException {
		// get all files with suffix of .dat
		File dir = new File("E:\\tmp\\");
		FilenameFilter ff = new FilenameFilter() {
			@Override
			public boolean accept(File dir, String name) {
				return name.startsWith("invt_lst_");
			}
		};
		File[] files = dir.listFiles(ff);

		File f1 = files[0];
		for (int i = 1; i < files.length; i++) {
			File tmp = new File("E:\\tmp\\"
					+ Calendar.getInstance().getTimeInMillis() + ".dat");
			merge(f1, files[i], tmp);
			if (i != 1)
				f1.delete();
			f1 = tmp;
		}
	}

	public void merge(File f1, File f2, File to) throws IOException {
		DataInputStream dis1 = new DataInputStream(new FileInputStream(f1));
		DataInputStream dis2 = new DataInputStream(new FileInputStream(f2));
		DataOutputStream dos = new DataOutputStream(new FileOutputStream(to));

		int c1 = -1;
		int c2 = -1;
		int l1 = -1;
		int l2 = -1;
		boolean eof1 = false;
		boolean eof2 = false;

		TreeSet<Integer> ts1 = new TreeSet<Integer>();
		TreeSet<Integer> ts2 = new TreeSet<Integer>();

		try {
			c1 = dis1.readInt();
			l1 = dis1.readInt();
			ts1.clear();
			for (int i = 0; i < l1; i++) {
				ts1.add(dis1.readInt());
			}
		} catch (EOFException e) {
			eof1 = true;
		}

		try {
			c2 = dis2.readInt();
			l2 = dis2.readInt();
			ts2.clear();
			for (int i = 0; i < l2; i++) {
				ts2.add(dis2.readInt());
			}
		} catch (EOFException e) {
			eof2 = true;
		}

		while (true) {

			System.out.print((char) c1);
			System.out.print((char) c2);
			System.out.println();

			if (eof1 && eof2) {
				break;

			} else if (!eof1 && !eof2 && c1 == c2) {
				ts1.addAll(ts2);

				dos.writeInt(c1);
				dos.writeInt(l1 + l2);

				Iterator<Integer> it = ts1.iterator();
				while (it.hasNext())
					dos.writeInt(it.next());

				try {
					c1 = dis1.readInt();
					l1 = dis1.readInt();
					ts1.clear();
					for (int i = 0; i < l1; i++) {
						ts1.add(dis1.readInt());
					}
				} catch (EOFException e) {
					eof1 = true;
				}

				try {
					c2 = dis2.readInt();
					l2 = dis2.readInt();
					ts2.clear();
					for (int i = 0; i < l2; i++) {
						ts2.add(dis2.readInt());
					}
				} catch (EOFException e) {
					eof2 = true;
				}

			} else if (eof1 || c1 < c2) {
				dos.writeInt(c1);
				dos.writeInt(l1);
				Iterator<Integer> it = ts1.iterator();
				while (it.hasNext())
					dos.writeInt(it.next());

				try {
					c1 = dis1.readInt();
					l1 = dis1.readInt();
					ts1.clear();
					for (int i = 0; i < l1; i++) {
						ts1.add(dis1.readInt());
					}
				} catch (EOFException e) {
					eof1 = true;
				}

			} else if (eof2 || c2 < c1) {
				dos.writeInt(c2);
				dos.writeInt(l2);
				Iterator<Integer> it = ts2.iterator();
				while (it.hasNext())
					dos.writeInt(it.next());

				try {
					c2 = dis2.readInt();
					l2 = dis2.readInt();
					ts2.clear();
					for (int i = 0; i < l2; i++) {
						ts2.add(dis2.readInt());
					}
				} catch (EOFException e) {
					eof2 = true;
				}
			}
		}

		dis1.close();
		dis2.close();
		dos.close();
	}

	public void checkChineseSimple(int startIdx) throws ClassNotFoundException,
			SQLException, IOException {
		int step = 2000;
		DBAccess db = new DBAccess();
		ResultSet rs;

		// get max id
		rs = db.executeQuery("select id from webpage order by id desc limit 1");
		int maxIdx = -1;

		if (rs.next())
			maxIdx = rs.getInt(1);
		else
			return;

		if (maxIdx == -1)
			return;

		// output file
		File f = new File("E:\\tmp\\id.txt");
		FileWriter fw = new FileWriter(f);
		BufferedWriter bw = new BufferedWriter(fw);
		PrintWriter pw = new PrintWriter(bw);

		//
		while (startIdx < maxIdx) {
			System.out.println("--> " + startIdx);
			rs.close();
			rs = db.executeQuery("select * from webpage where id > " + startIdx
					+ " limit " + step);

			while (rs.next()) {
				// int docId = rs.getInt("docId");
				String text = rs.getString("text");
				startIdx = rs.getInt("id");

				if (text == null)
					return;

				text = text.replaceAll("[^\u4e00-\u9fa5]", "");

				int len = text.length();
				int nonSimple = 0;
				for (int i = 0; i < len; i++) {
					char c = text.charAt(i);
					if (!isChineseSimple(c))
						nonSimple++;
				}

				if (nonSimple > 0 && len / nonSimple < 5)
					pw.write(startIdx + ", ");
			}

			System.out.println("<-- " + startIdx);
		}
	}

	private char[] ARR = STR.COMM_HZ.toCharArray();
	private int LEN = STR.COMM_HZ.length();

	private boolean isChineseSimple(char c) {
		int h = LEN - 1, l = 0, m;

		while (l <= h) {
			m = (h + l) / 2;
			if (c > ARR[m]) {
				l = m + 1;
			} else if (c < ARR[m]) {
				h = m - 1;
			} else {
				return true;
			}
		}

		return false;
	}

	public void CreateIndex(int startIdx) throws ClassNotFoundException,
			SQLException, IOException {
		Map<Character, HashSet<Integer>> invt = new HashMap<Character, HashSet<Integer>>();
		ResultSet rs;
		DBAccess db = new DBAccess();
		rs = db.executeQuery("select id from webpage order by id desc limit 1");
		int maxIdx = -1;
		int step = 10000;

		if (rs.next())
			maxIdx = rs.getInt(1);
		else
			return;

		if (maxIdx == -1)
			return;

		String text = null;
		Set<Character> uniqueToken = new HashSet<Character>();
		uniqueToken.clear();
		int docId = -1;
		int s = startIdx;

		while (startIdx < maxIdx) {
			s = startIdx;
			System.out.println(s);
			rs.close();
			rs = db.executeQuery("select * from webpage where id > " + startIdx
					+ " limit " + step);

			invt.clear();
			while (rs.next()) {
				docId = rs.getInt("docId");
				text = rs.getString("text");
				startIdx = rs.getInt("id");

				if (text == null)
					continue;

				text = text.replaceAll("[^\u4e00-\u9fa5]", "");

				for (int i = 0; i < text.length(); i++) {
					char c = text.charAt(i);

					if (!isChineseSimple(c))
						continue;

					if (invt.containsKey(c)) {
						invt.get(c).add(docId);
					} else {
						HashSet<Integer> lst = new HashSet<Integer>();
						lst.add(docId);
						invt.put(c, lst);
					}
				}
			}

			// out put invert list
			File f = new File(String.format("E:\\tmp\\invt_lst_%08d.dat", s));
			FileOutputStream fos = new FileOutputStream(f, true);
			DataOutputStream dos = new DataOutputStream(fos);
			Iterator<Character> keys = invt.keySet().iterator();

			while (keys.hasNext()) {
				char c = keys.next();
				HashSet<Integer> lst = invt.get(c);
				int sz = lst.size();
				if (sz == 0)
					continue;

				dos.writeInt((int) c);
				dos.writeInt(sz);

				for (int num : lst)
					dos.writeInt(num);
			}

			dos.close();

			// debug
			// break;
		}

	}

	public void multithreadCrawler(int startIdx) throws ClassNotFoundException,
			SQLException {
		Dispatcher disp = new Dispatcher();
		DBAccess db = new DBAccess();
		ResultSet rs = db.executeQuery("select count(*) from webpage");
		int rcdNum = 0;
		int idx = startIdx;
		int step = 100;

		if (rs.next())
			rcdNum = rs.getInt(1);
		else
			return;

		String url;
		int id;

		while (idx < rcdNum) {
			System.out.println(idx);
			rs = db.executeQuery("select * from webpage limit " + idx + ","
					+ step);

			while (rs.next()) {
				url = rs.getString("link");
				if (!url.startsWith("http"))
					url = "http://" + url;
				id = rs.getInt("id");

				JobInfo ji = new JobInfo(id, url);
				while (disp.getJobNum() > 50)
					try {
						Thread.sleep(300);
					} catch (InterruptedException e) {
						e.printStackTrace();
					}
				disp.run(ji);
			}
			rs.last();
			idx += rs.getRow();
		}

		while (disp.getJobNum() > 0) {
			try {
				Thread.sleep(1000);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}

		disp.stop();
	}

	public void update_rcd_of_id(int id) throws ClassNotFoundException,
			SQLException, IOException {
		DBAccess db = new DBAccess();
		ResultSet rs = db.executeQuery("select * from webpage where id=" + id);
		String url = null;
		if (rs.next()) {
			url = rs.getString("link");
		} else
			return;

		if (url == null)
			return;

		Crawler c = new Crawler();
		System.out.println(url);
		Map<String, String> rst = c.getPage(url);
		System.out.println(rst.get("enc"));
		String txt = rst.get("txt");
		txt = txt.replace("'", "''");
		txt = txt.replace("\\", "\\\\");
		String sql = String.format("update webpage set text='%s' where id=%d",
				txt, id);
		System.out.println(sql);
		db.execute(sql);
	}

	public void crawler(int startIdx) throws ClassNotFoundException,
			SQLException {
		DBAccess db = new DBAccess();
		ResultSet rs = db.executeQuery("select count(*) from webpage");
		int rcdNum = 0;
		int idx = startIdx;
		int step = 100;

		if (rs.next())
			rcdNum = rs.getInt(1);
		else
			return;

		System.out.println("total records: " + rcdNum);

		Crawler c = new Crawler();
		String url;
		String text = null;
		String enc = null;
		int id;
		String sql;
		Map<String, String> rst = null;

		while (idx < rcdNum) {
			System.out.println(idx);
			rs = db.executeQuery("select * from webpage limit " + idx + ","
					+ step);

			while (rs.next()) {
				url = rs.getString("link");
				if (!url.startsWith("http"))
					url = "http://" + url;
				id = rs.getInt("id");
				try {
					rst = c.getPage(url);
					if (rst == null)
						continue;
					text = rst.get("txt");
					enc = rst.get("enc");

				} catch (IOException ioe) {
					System.out.println(">>>");
					System.out.println(enc + ", " + id);
					System.out.println(url);
					System.out.println(text);
					System.out.println(rst == null ? "null" : rst.get("rsp"));
					ioe.printStackTrace();
					System.out.println("<<<");
					continue;
				}

				if (text == null)
					continue;
				text = text.replace("'", "''");
				text = text.replace("\\", "\\\\");

				sql = String.format("update webpage set text='%s' where id=%d",
						text, id);

				// System.out.println(sql);

				try {
					// db.setEncoding(enc);
					db.execute(sql);
				} catch (SQLException sqle) {
					System.out.println(id);
					System.out.println(sqle.getMessage());
					continue;
				}
			}
			rs.last();
			idx += rs.getRow();
		}
	}

	public void insertQueryLog(int lNo) throws ClassNotFoundException,
			SQLException, FileNotFoundException {
		DBAccess db = new DBAccess();

		FileInputStream fis = new FileInputStream(data_dir
				+ "\\stripped.sample");
		Scanner scanner = new Scanner(fis, "gbk");
		String line;

		int lineNo = 0;
		// String text = null;
		String url;
		String query;

		// Crawler c = new Crawler();

		while (scanner.hasNextLine()) {
			line = scanner.nextLine();
			lineNo++;
			if (lineNo < lNo)
				continue;
			System.out.println("line NO: " + lineNo);

			if (!line.contains(","))
				continue;

			query = line.split(",")[0];
			url = line.split(",")[1];

			if (query.length() < 1)
				continue;
			query = query.replace("'", "''");
			query = query.replace("\\", "\\\\");

			// shortest domain "g.cn" contains 4 characters
			if (url.length() < 4)
				continue;
			url = url.replace("'", "''");
			url = url.replace("\\", "\\\\");

			// add protocol, defaults to http
			if (!url.contains("://"))
				url = "http://" + url;

			String sql = String.format("insert into webpage "
					+ "(query, link, lineNo) values ('%s', '%s', %d)", query,
					url, lineNo);
			try {
				db.execute(sql);
			} catch (SQLException e) {
				System.out.println(sql);
				throw e;
			}
			// try {
			// text = c.getPage(url);
			// } catch (IOException ioe) {
			// System.out.println(ioe.getMessage());
			// continue;
			// }
			//
			// if (text == null)
			// continue;
		}
		scanner.close();
	}

	public void Index() throws IOException {
		FileInputStream fis = new FileInputStream(data_dir
				+ "\\stripped.sample");
		Scanner scanner = new Scanner(fis, "gbk");
		String line;

		DataOutputStream dos = new DataOutputStream(new FileOutputStream(
				new File("D:\\data\\text.dat"), true));

		int docId = 0;
		int lineNo = 0;
		String text = null;
		String url;

		Crawler c = new Crawler();
		SmartChineseAnalyzer analyzer = new SmartChineseAnalyzer(
				Version.LUCENE_31);
		Indexer idx = new Indexer();
		StringReader reader;
		TokenStream ts;
		CharTermAttribute cta;
		Map<String, String> rst = null;

		while (scanner.hasNextLine()) {
			line = scanner.nextLine();
			lineNo++;
			System.out.println("line NO: " + lineNo);

			if (!line.contains(","))
				continue;

			url = line.split(",")[1];

			// shortest domain "g.cn" contains 4 characters
			if (url.length() < 4)
				continue;

			// add protocol, defaults to http
			if (!url.contains("://"))
				url = "http://" + url;

			try {
				rst = c.getPage(url);
				if (rst == null)
					continue;
				text = rst.get("txt");
			} catch (IOException ioe) {
				System.out.println(ioe.getMessage());
				continue;
			}

			if (text == null)
				continue;

			// save crawled text and line NO
			dos.writeInt(Integer.SIZE / 8 * 2 + text.getBytes().length);
			dos.writeInt(lineNo);
			dos.writeBytes(text);

			reader = new StringReader(text);
			ts = analyzer.tokenStream("content", reader);
			if (ts == null)
				continue;
			cta = ts.getAttribute(CharTermAttribute.class);
			while (ts.incrementToken()) {
				idx.add(cta.toString(), docId);
			}
			docId++;
			System.out.println("docId: " + docId);
		}

		// out put inverted index
		idx.dump();

		scanner.close();
		dos.close();
	}

	public void step1() throws Exception {
		File f = new File("E:\\workspace\\data\\SogouQ.reduced");

		if (!f.exists())
			return;
		FileInputStream fis = new FileInputStream(f);
		Scanner scanner = new Scanner(fis, "gbk");
		String line;
		File fout = new File("E:\\workspace\\data\\reduced.sample");
		FileOutputStream fos = new FileOutputStream(fout);
		Writer out = new OutputStreamWriter(fos, "gbk");

		while (scanner.hasNextLine()) {
			line = scanner.nextLine();
			String[] s = line.split("[\\ \\t]");
			if (s.length == 6) {
				out.write(s[2].substring(1, s[2].length() - 1) + "," + s[5]
						+ System.getProperty("line.separator"));
			}
		}

		out.close();
		scanner.close();
		System.out.println("over");
	}

	public void step2() throws Exception {
		File f = new File("E:\\workspace\\data\\reduced.sample");

		if (!f.exists())
			return;
		FileInputStream fis = new FileInputStream(f);
		Scanner scanner = new Scanner(fis, "gbk");
		String line;
		File fout = new File("E:\\workspace\\data\\stripped.sample");
		FileOutputStream fos = new FileOutputStream(fout);
		Writer out = new OutputStreamWriter(fos, "gbk");

		HashSet<String> hset = new HashSet<String>();
		while (scanner.hasNextLine()) {
			line = scanner.nextLine();
			String[] s = line.split(",");

			if (s.length == 2) {
				if (hset.contains(s[1]))
					continue;
				hset.add(s[1]);

				out.write(s[0] + "," + s[1]
						+ System.getProperty("line.separator"));
			}
		}

		out.close();
		scanner.close();
		System.out.println("over");
	}
}
